package com.example.myFirstWebmagic.demo;

import java.util.ArrayList;
import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import com.google.gson.JsonObject;

/**
 * 
 * ClassName: CarPhone
 * 
 * @Description: 爬取汽车4S店电话
 * @author xiongyao
 * @date 2018年1月2日
 */
public class CarPhoneYiMao implements PageProcessor {

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(10).setSleepTime(5000);

	public int pageSize = 20;

	public int nowPage = 1;

	public List<JsonObject> fourServices = new ArrayList<JsonObject>();

	public static JsonObject jsonObject = new JsonObject();

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String[] startUrl = new String[] { "http://dealer.emao.com/wuhan/list-0-0-0_1.html" };
		Spider.create(new CarPhoneYiMao()).addUrl(startUrl)
				.addPipeline(new ConsolePipeline())
				.addPipeline(new FilePipeline()).thread(5).run();
		System.out.println(jsonObject.toString());
	}

	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
		List<String> names = page
				.getHtml()
				.xpath("/html/body/div[4]/div[1]/div[3]/div[2]/*/div[2]/h6/p/a/html()")
				.all();
		// names.forEach(item -> System.out.println(item));

		// page.putField("4Sname", names);

		List<String> phones = page
				.getHtml()
				.xpath("/html/body/div[4]/div[1]/div[3]/div[2]/*/div[2]/p[2]/span/html()")
				.all();
		// phones.forEach(item -> System.out.println(item));

		// page.putField("4Sphone", phones);

		for (int i = 0; i < names.size(); i++) {
			jsonObject.addProperty(names.get(i), phones.get(i));
		}

		// if (nowPage <= pageSize) {
		// page.addTargetRequest("https://dealer.autohome.com.cn/wuhan?countyId=0&brandId=0&seriesId=0&factoryId=0&pageIndex="
		// + nowPage + "&kindId=1&orderType=0&isSales=0");
		// } else {
		// System.out.println(jsonObject.toString());
		// }

		// System.out.println(page.getHtml()
		// .xpath("/html/body/div[4]/div[1]/div[4]/ul/*/a/@href").all());

		page.addTargetRequests(page.getHtml()
				.xpath("/html/body/div[4]/div[1]/div[4]/ul/*/a/@href").all());

	}

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

}
